{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b1cd6593-e1ba-4a0d-a0fe-d95955f7d474",
   "metadata": {},
   "source": [
    "https://scikit-learn.org/1.1/modules/svm.html#classification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c92a76bc-3b1f-482f-abad-394a8a09d5b4",
   "metadata": {},
   "source": [
    "## Training and Logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1b0edafd-e9ff-41f5-bf3c-ef6cad44cbef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((569, 30), (569,))"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "X, y = load_breast_cancer(return_X_y=True)\n",
    "(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a25767fc-b34f-45f2-8d81-7ffe4bfc166c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(array([1.799e+01, 1.038e+01, 1.228e+02, 1.001e+03, 1.184e-01, 2.776e-01,\n",
       "         3.001e-01, 1.471e-01, 2.419e-01, 7.871e-02, 1.095e+00, 9.053e-01,\n",
       "         8.589e+00, 1.534e+02, 6.399e-03, 4.904e-02, 5.373e-02, 1.587e-02,\n",
       "         3.003e-02, 6.193e-03, 2.538e+01, 1.733e+01, 1.846e+02, 2.019e+03,\n",
       "         1.622e-01, 6.656e-01, 7.119e-01, 2.654e-01, 4.601e-01, 1.189e-01]),\n",
       "  0),\n",
       " (array([2.057e+01, 1.777e+01, 1.329e+02, 1.326e+03, 8.474e-02, 7.864e-02,\n",
       "         8.690e-02, 7.017e-02, 1.812e-01, 5.667e-02, 5.435e-01, 7.339e-01,\n",
       "         3.398e+00, 7.408e+01, 5.225e-03, 1.308e-02, 1.860e-02, 1.340e-02,\n",
       "         1.389e-02, 3.532e-03, 2.499e+01, 2.341e+01, 1.588e+02, 1.956e+03,\n",
       "         1.238e-01, 1.866e-01, 2.416e-01, 1.860e-01, 2.750e-01, 8.902e-02]),\n",
       "  0)]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(X[0], y[0]), (X[1], y[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cf9ee029-703c-4972-a077-19ca950e866a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/da/.cache/pants/named_caches/pex_root/venvs/s/378f4125/venv/lib/python3.9/site-packages/sklearn/svm/_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n",
      "/home/da/.cache/pants/named_caches/pex_root/venvs/s/378f4125/venv/lib/python3.9/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\n",
      "  warnings.warn(\"Setuptools is replacing distutils.\")\n",
      "2023/03/29 13:13:52 WARNING mlflow.utils.environment: Failed to resolve installed pip version. ``pip`` will be added to conda.yaml environment spec without a version specifier.\n",
      "Registered model 'da_svc_clf' already exists. Creating a new version of this model...\n",
      "2023/03/29 13:13:52 INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation.                     Model name: da_svc_clf, version 2\n",
      "Created version '2' of model 'da_svc_clf'.\n",
      "/home/da/.cache/pants/named_caches/pex_root/venvs/s/378f4125/venv/lib/python3.9/site-packages/liga/mlflow/logger.py:137: UserWarning: value of rikai.output.schema is None or empty and will not be populated to MLflow\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import getpass\n",
    "\n",
    "import mlflow\n",
    "from liga.sklearn.mlflow import log_model\n",
    "from sklearn import svm\n",
    "\n",
    "\n",
    "mlflow_tracking_uri = \"sqlite:///mlruns.db\"\n",
    "mlflow.set_tracking_uri(mlflow_tracking_uri)\n",
    "\n",
    "# train a model\n",
    "with mlflow.start_run() as run:\n",
    "    ####\n",
    "    # Part 1: Train the model and register it on MLflow\n",
    "    ####\n",
    "    \n",
    "    # model = svm.SVC().fit(X, y)\n",
    "    # model = svm.NuSVC().fit(X, y)\n",
    "    model = svm.LinearSVC().fit(X, y)\n",
    "    \n",
    "    registered_model_name = f\"{getpass.getuser()}_svc_clf\"\n",
    "    log_model(model, registered_model_name=registered_model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a6af181-7599-4dcd-a583-a9aa9f8b645a",
   "metadata": {},
   "source": [
    "## Apply the model on the large scale dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b49d7efc-5314-4efe-af40-c63f31b6bfc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-29 13:13:52,764 INFO Rikai (__init__.py:121): setting spark.sql.extensions to net.xmacs.liga.spark.RikaiSparkSessionExtensions\n",
      "2023-03-29 13:13:52,764 INFO Rikai (__init__.py:121): setting spark.driver.extraJavaOptions to -Dio.netty.tryReflectionSetAccessible=true\n",
      "2023-03-29 13:13:52,765 INFO Rikai (__init__.py:121): setting spark.executor.extraJavaOptions to -Dio.netty.tryReflectionSetAccessible=true\n",
      "2023-03-29 13:13:52,765 INFO Rikai (__init__.py:121): setting spark.jars to https://github.com/komprenilo/liga/releases/download/v0.3.0/liga-spark321-assembly_2.12-0.3.0.jar\n",
      "23/03/29 13:13:53 WARN Utils: Your hostname, tubi resolves to a loopback address: 127.0.1.1; using 192.168.31.32 instead (on interface wlp0s20f3)\n",
      "23/03/29 13:13:53 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "23/03/29 13:13:58 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    }
   ],
   "source": [
    "from example import spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ef3b8266-6189-4d77-aab1-cfb6dc5a4162",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------+------+--------------------+-------+\n",
      "|name            |plugin|uri                 |options|\n",
      "+----------------+------+--------------------+-------+\n",
      "|mlflow_sklearn_m|      |mlflow:///da_svc_clf|       |\n",
      "+----------------+------+--------------------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from liga.mlflow import CONF_MLFLOW_TRACKING_URI\n",
    "spark.conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"false\")\n",
    "spark.conf.set(CONF_MLFLOW_TRACKING_URI, mlflow_tracking_uri)\n",
    "spark.sql(f\"\"\"\n",
    "CREATE OR REPLACE MODEL mlflow_sklearn_m LOCATION 'mlflow:///{registered_model_name}';\n",
    "\"\"\"\n",
    ")\n",
    "\n",
    "spark.sql(\"show models\").show(1, vertical=False, truncate=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cc7c75cd-3e9d-4de5-accc-917056503188",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- mlflow_sklearn_m: integer (nullable = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mlflow_sklearn_m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mlflow_sklearn_m\n",
       "0                 0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = spark.sql(f\"\"\"\n",
    "select ML_PREDICT(mlflow_sklearn_m, array(1.799e+01, 1.038e+01, 1.228e+02, 1.001e+03, 1.184e-01, 2.776e-01,\n",
    "        3.001e-01, 1.471e-01, 2.419e-01, 7.871e-02, 1.095e+00, 9.053e-01,\n",
    "        8.589e+00, 1.534e+02, 6.399e-03, 4.904e-02, 5.373e-02, 1.587e-02,\n",
    "        3.003e-02, 6.193e-03, 2.538e+01, 1.733e+01, 1.846e+02, 2.019e+03,\n",
    "        1.622e-01, 6.656e-01, 7.119e-01, 2.654e-01, 4.601e-01, 1.189e-01))\n",
    "\"\"\"\n",
    ")\n",
    "\n",
    "result.printSchema()\n",
    "result.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "12b62a63-fc18-455f-ba33-d37ce159bb88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- mlflow_sklearn_m: integer (nullable = true)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mlflow_sklearn_m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mlflow_sklearn_m\n",
       "0                 1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = spark.sql(f\"\"\"\n",
    "select ML_PREDICT(mlflow_sklearn_m, array(7.760e+00, 2.454e+01, 4.792e+01, 1.810e+02, 5.263e-02, 4.362e-02,\n",
    "        0.000e+00, 0.000e+00, 1.587e-01, 5.884e-02, 3.857e-01, 1.428e+00,\n",
    "        2.548e+00, 1.915e+01, 7.189e-03, 4.660e-03, 0.000e+00, 0.000e+00,\n",
    "        2.676e-02, 2.783e-03, 9.456e+00, 3.037e+01, 5.916e+01, 2.686e+02,\n",
    "        8.996e-02, 6.444e-02, 0.000e+00, 0.000e+00, 2.871e-01, 7.039e-02))\n",
    "\"\"\"\n",
    ")\n",
    "\n",
    "result.printSchema()\n",
    "result.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c8e843e-3ec9-42c6-859b-8e4edca567d4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
